#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
@Project ：pyspark 
@File    ：pyspark_test.py
@IDE     ：PyCharm 
@Author  ：Amos.Mu
@Date    ：2022/5/14 10:03 
'''

from pyspark import SparkContext
import os

PYSPARK_PYTHON = "/root/anaconda3/envs/pyspark/bin/python3"
os.environ["PYSPARK_PYTHON"] = PYSPARK_PYTHON
os.environ["PYSPARK_DRIVER_PYTHON"] = PYSPARK_PYTHON

sc = SparkContext("local[*]", "wordcount")
words = sc.textFile('file:///root/word.txt') \
    .flatMap(lambda line: line.split(" ")) \
    .map(lambda x: (x, 1)) \
    .reduceByKey(lambda a, b: a + b).collect()
print(words)
